package com.example.origin.spider.demo;

import lombok.SneakyThrows;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.DuplicateRemovedScheduler;
import us.codecraft.webmagic.scheduler.QueueScheduler;

import java.util.ArrayList;

public class WebmagicDemo implements PageProcessor {
    //主函数，执行爬虫
    @SneakyThrows
    public static void main(String[] args) {
        // 代理ip
        HttpClientDownloader downloader = new HttpClientDownloader(){
            @Override
            protected void onError(Page page, Task task, Throwable e) {
                // todo
            }
        };
        Proxy proxy = new Proxy("111.26.177.28", 9091,"http");
        SimpleProxyProvider proxyProvider = SimpleProxyProvider.from(proxy);
//        ArrayList<Proxy> proxies = new ArrayList<>();
//        proxies.add(proxy);
//        SimpleProxyProvider proxyProvider = new SimpleProxyProvider(proxies) ;

        downloader.setProxyProvider(proxyProvider);


        // url 去重 重复的一定找到，不重复的会误判
        DuplicateRemovedScheduler scheduler = new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(1000000));
        // todo 网页内容去重，simhash

        Request request = new Request();
        request.setDownloader(downloader);
//        request.setUrl("https://www.jd.com/moreSubject.aspx");
        request.setUrl("https://www.baidu.com");
        request.setCharset("utf-8");

        Spider spider = Spider.create(new WebmagicDemo())
                .addRequest(request)
                //.addUrl("https://www.jd.com/moreSubject.aspx")//设置取数据的页面
                .addPipeline(new ConsolePipeline())
                .thread(2)
                .setDownloader(downloader)
                .setScheduler(scheduler);
        spider.run();//执行
        // 添加监控
        SpiderMonitor.instance().register(spider);
    }

    //解折页面
    public void process(Page page) {
        System.out.println("=======================>>"+page.getUrl());
        if (page.getUrl().get().contains("ip.chaxunjiao.com")){
            page.putField("ip", page.getHtml().css("h3").get());
            return;
        }else {
            //解折返回的数据page,并目把解折的结果放到ResultItems中
            page.putField("div", page.getHtml().css("div.dt a").all());
        }

        page.addTargetRequest("https://ip.chaxunjiao.com/");
    }

    public Site getSite() {
        Site site = Site.me();
        site.setCharset("utf-8");
        site.setTimeOut(10000);//设置超时时间，单位是ms毫秒
        site.setRetrySleepTime(3000);//设置重试的间时间
        site.setSleepTime(3);//设置重试次数
        return site;
    }


}
